# -*- coding: utf-8 -*-
import scrapy
import re
from PhoneSpider.items import PhonespiderItem
from pandas import DataFrame
import pandas as pd
import json
import time
class phonespider(scrapy.spiders.Spider):
    name = 'ZolSpider'
    allowed_domain = {'zol.com.cn'}
    start_urls = {
        'http://detail.zol.com.cn/cell_phone_index/subcate57_0_list_1_0_1_2_0_1.html'
    }
    sheet = 1
    data = list()
    count = 0
    phone = {}
    phone1 = {}
    count = 0
    phoneNum= 0
    def parse(self,response):
        #获取页码数
        for sel in response.xpath('//*[@id="J_PicMode"]/li'):
            url = "http://detail.zol.com.cn%s"  % sel.xpath('h3/a/@href').extract()[0]
            name = sel.xpath('h3/a/text()').extract()[0].encode('utf-8')
            price = ''.join(sel.xpath('div[@class="price-row"]/span[@class="price price-normal"]').xpath('string(.)').extract())
            print price
            item = PhonespiderItem()
            item['price'] = price
            item['name'] = name

            yield scrapy.Request(url, callback=self.parse1,meta={'item': item})

        if self.count<305:
            self.count = self.count+1
            try:
                nexturl = "http://detail.zol.com.cn%s" % response.xpath('//*[@class="page-box"]/div[@class="pagebar"]/a[@class="next"]/@href').extract()[0]
                yield scrapy.Request(nexturl, callback=self.parse)
            except:
                pd.concat(self.data).to_excel('zolphone%f.xlsx' % time.time())
                print self.phoneNum
                print "End Scrawl"
        else:
            pd.concat(self.data).to_excel('zolphone%f.xlsx' % time.time())
            print "End Scrawl"
            print self.phoneNum
            return

    def parse1(self,response):
        url = "http://detail.zol.com.cn%s" % response.xpath('//*[@id="tagNav"]/ul/li[4]/a/@href').extract()[0]
        item = response.meta['item']
        yield scrapy.Request(url, callback=self.parse2,meta={'item': item})

    def parse2(self,response):
        self.phone = {}
        name = response.xpath('//*[@id="page-title"]/text()').extract()[0].encode('utf-8')
        self.phone['Name'] = name
        self.phone['NameO'] = response.meta['item']['name']
        self.phone['Price'] = response.meta['item']['price']
        for i in range(len(response.xpath('//*[@class="param-name"]'))):
            pname = response.xpath('//*[@id="newPmName_%d"]/text()' % int(i+1)).extract()[0].encode('utf-8')
            valsuit = response.xpath('//*[@id="newPmVal_%d"]' % int(i+1))[0]
            info = ''.join(valsuit.xpath('string(.)').extract()).encode('utf-8')
            self.phone[pname] = info
        self.phoneNum = self.phoneNum + 1
        print DataFrame(self.phone,index=[name])
        print self.phoneNum
        self.data.append(DataFrame(self.phone,index=[name]))
